{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# 第四部分 模型篇 —— 模型调参"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 利用Ray-Tune自动调优XGBoost模型"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "1）导入必要的包"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'0.8.4'"
      ]
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "import numpy as np\n",
    "import ray\n",
    "from ray import tune\n",
    "import xgboost as xgb\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "ray.__version__"
   ]
  },
  {
   "source": [
    "2）定义回调函数和优化目标"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def XGBCallback(env):\n",
    "    tune.track.log(**dict(env.evaluation_result_list))\n",
    "\n",
    "def train_breast_cancer(config):\n",
    "    # 每次随机划分\n",
    "    data, target = load_breast_cancer(return_X_y=True)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(data, target)\n",
    "\n",
    "    train_set = xgb.DMatrix(X_train, label=y_train)\n",
    "    test_set = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "    bst = xgb.train(config, train_set, evals=[(test_set, 'eval')], callbacks=[XGBCallback])\n",
    "\n",
    "    preds = bst.predict(test_set)\n",
    "    pred_labels = np.rint(preds)\n",
    "\n",
    "    tune.track.log(mean_accuracy=accuracy_score(y_test, pred_labels), done=True)\n"
   ]
  },
  {
   "source": [
    "3）定义超参空间"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'verbosity': 0,\n",
    "    'num_threads': 2,\n",
    "    'objective': 'binary:logistic',\n",
    "    'booster': 'gbtree',\n",
    "    # 每轮输出多种评估指标\n",
    "    'eval_metric': ['auc', 'ams@0', 'logloss'],\n",
    "    'max_depth': tune.randint(1, 9),\n",
    "    'eta': tune.loguniform(1e-4, 1e-1),\n",
    "    'gamma': tune.loguniform(1e-8, 1.0),\n",
    "    'grow_policy': tune.choice(['depthwise', 'lossguide'])\n",
    "}\n"
   ]
  },
  {
   "source": [
    "4）定义Tune优化器并运行"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "2021-02-27 19:29:50,452\tINFO resource_spec.py:212 -- Starting Ray with 333.79 GiB memory available for workers and up to 147.05 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).\n",
      "2021-02-27 19:29:50,924\tINFO services.py:1148 -- View the Ray dashboard at \u001b[1m\u001b[32mlocalhost:8265\u001b[39m\u001b[22m\n",
      "2021-02-27 19:29:51,612\tWARNING tune.py:312 -- Tune detects GPUs, but no trials are using GPUs. To enable trials to use GPUs, set tune.run(resources_per_trial={'gpu': 1}...) which allows Tune to expose 1 GPU to each trial. You can also override `Trainable.default_resource_request` if using the Trainable API.\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "== Status ==<br>Memory usage on this node: 18.0/503.2 GiB<br>Using AsyncHyperBand: num_stopped=0\nBracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: None<br>Resources requested: 2/48 CPUs, 0/1 GPUs, 0.0/333.79 GiB heap, 0.0/101.46 GiB objects<br>Result logdir: /home/xuhaowei/ray_results/train_breast_cancer<br>Number of trials: 3 (2 PENDING, 1 RUNNING)<br><table>\n<thead>\n<tr><th>Trial name               </th><th>status  </th><th>loc  </th><th style=\"text-align: right;\">        eta</th><th style=\"text-align: right;\">      gamma</th><th>grow_policy  </th><th style=\"text-align: right;\">  max_depth</th></tr>\n</thead>\n<tbody>\n<tr><td>train_breast_cancer_00000</td><td>RUNNING </td><td>     </td><td style=\"text-align: right;\">0.0363721  </td><td style=\"text-align: right;\">1.70701e-05</td><td>depthwise    </td><td style=\"text-align: right;\">          4</td></tr>\n<tr><td>train_breast_cancer_00001</td><td>PENDING </td><td>     </td><td style=\"text-align: right;\">0.0111021  </td><td style=\"text-align: right;\">0.00706989 </td><td>lossguide    </td><td style=\"text-align: right;\">          5</td></tr>\n<tr><td>train_breast_cancer_00002</td><td>PENDING </td><td>     </td><td style=\"text-align: right;\">0.000227598</td><td style=\"text-align: right;\">4.42244e-07</td><td>lossguide    </td><td style=\"text-align: right;\">          8</td></tr>\n</tbody>\n</table><br><br>"
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m 2021-02-27 19:29:52,731\tINFO trainable.py:217 -- Getting current IP.\n",
      "\u001b[2m\u001b[36m(pid=20241)\u001b[0m 2021-02-27 19:29:52,798\tINFO trainable.py:217 -- Getting current IP.\n",
      "\u001b[2m\u001b[36m(pid=20208)\u001b[0m 2021-02-27 19:29:52,871\tINFO trainable.py:217 -- Getting current IP.\n",
      "Result for train_breast_cancer_00000:\n",
      "  date: 2021-02-27_19-29-52\n",
      "  done: false\n",
      "  eval-ams@0: 15.206998\n",
      "  eval-auc: 0.991336\n",
      "  eval-logloss: 0.663255\n",
      "  experiment_id: a29a9a5873684c0a98715b53c7372037\n",
      "  experiment_tag: 0_eta=0.036372,gamma=1.707e-05,grow_policy=depthwise,max_depth=4\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20202\n",
      "  time_since_restore: 0.22225594520568848\n",
      "  time_this_iter_s: 0.22225594520568848\n",
      "  time_total_s: 0.22225594520568848\n",
      "  timestamp: 1614425392\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 0\n",
      "  trial_id: '00000'\n",
      "  \n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [0]\teval-auc:0.99134\teval-ams@0:15.20700\teval-logloss:0.66326\n",
      "Result for train_breast_cancer_00001:\n",
      "  date: 2021-02-27_19-29-53\n",
      "  done: false\n",
      "  eval-ams@0: 13.697143\n",
      "  eval-auc: 0.961415\n",
      "  eval-logloss: 0.683942\n",
      "  experiment_id: 42de8b0280be4224b6e56807d6410fc8\n",
      "  experiment_tag: 1_eta=0.011102,gamma=0.0070699,grow_policy=lossguide,max_depth=5\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20241\n",
      "  time_since_restore: 0.4533872604370117\n",
      "  time_this_iter_s: 0.4533872604370117\n",
      "  time_total_s: 0.4533872604370117\n",
      "  timestamp: 1614425393\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 0\n",
      "  trial_id: '00001'\n",
      "  \n",
      "\u001b[2m\u001b[36m(pid=20241)\u001b[0m [0]\teval-auc:0.96142\teval-ams@0:13.69714\teval-logloss:0.68394\n",
      "Result for train_breast_cancer_00002:\n",
      "  date: 2021-02-27_19-29-53\n",
      "  done: false\n",
      "  eval-ams@0: 15.136933\n",
      "  eval-auc: 0.947584\n",
      "  eval-logloss: 0.692961\n",
      "  experiment_id: 3913e28c38454c29bd117a35cc923add\n",
      "  experiment_tag: 2_eta=0.0002276,gamma=4.4224e-07,grow_policy=lossguide,max_depth=8\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20208\n",
      "  time_since_restore: 0.4338545799255371\n",
      "  time_this_iter_s: 0.4338545799255371\n",
      "  time_total_s: 0.4338545799255371\n",
      "  timestamp: 1614425393\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 0\n",
      "  trial_id: '00002'\n",
      "  \n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [1]\teval-auc:0.99112\teval-ams@0:15.20700\teval-logloss:0.63573\n",
      "\u001b[2m\u001b[36m(pid=20208)\u001b[0m [0]\teval-auc:0.94758\teval-ams@0:15.13693\teval-logloss:0.69296\n",
      "Result for train_breast_cancer_00001:\n",
      "  date: 2021-02-27_19-29-53\n",
      "  done: true\n",
      "  eval-ams@0: 13.565317\n",
      "  eval-auc: 0.961915\n",
      "  eval-logloss: 0.675459\n",
      "  experiment_id: 42de8b0280be4224b6e56807d6410fc8\n",
      "  experiment_tag: 1_eta=0.011102,gamma=0.0070699,grow_policy=lossguide,max_depth=5\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 2\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20241\n",
      "  time_since_restore: 0.7931804656982422\n",
      "  time_this_iter_s: 0.33979320526123047\n",
      "  time_total_s: 0.7931804656982422\n",
      "  timestamp: 1614425393\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: '00001'\n",
      "  \n",
      "Result for train_breast_cancer_00002:\n",
      "  date: 2021-02-27_19-29-53\n",
      "  done: true\n",
      "  eval-ams@0: 15.136933\n",
      "  eval-auc: 0.947584\n",
      "  eval-logloss: 0.692776\n",
      "  experiment_id: 3913e28c38454c29bd117a35cc923add\n",
      "  experiment_tag: 2_eta=0.0002276,gamma=4.4224e-07,grow_policy=lossguide,max_depth=8\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 2\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20208\n",
      "  time_since_restore: 0.7695767879486084\n",
      "  time_this_iter_s: 0.3357222080230713\n",
      "  time_total_s: 0.7695767879486084\n",
      "  timestamp: 1614425393\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: '00002'\n",
      "  \n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [2]\teval-auc:0.99091\teval-ams@0:15.20700\teval-logloss:0.61019\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [3]\teval-auc:0.99091\teval-ams@0:15.20700\teval-logloss:0.58616\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [4]\teval-auc:0.99091\teval-ams@0:15.20700\teval-logloss:0.56349\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [5]\teval-auc:0.99134\teval-ams@0:15.20700\teval-logloss:0.54275\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [6]\teval-auc:0.99134\teval-ams@0:15.20700\teval-logloss:0.52296\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [7]\teval-auc:0.99134\teval-ams@0:15.20700\teval-logloss:0.50474\n",
      "\u001b[2m\u001b[36m(pid=20202)\u001b[0m [8]\teval-auc:0.99134\teval-ams@0:15.20700\teval-logloss:0.48687\n",
      "Result for train_breast_cancer_00000:\n",
      "  date: 2021-02-27_19-29-54\n",
      "  done: true\n",
      "  experiment_id: a29a9a5873684c0a98715b53c7372037\n",
      "  experiment_tag: 0_eta=0.036372,gamma=1.707e-05,grow_policy=depthwise,max_depth=4\n",
      "  hostname: cieserver\n",
      "  iterations_since_restore: 11\n",
      "  mean_accuracy: 0.9370629370629371\n",
      "  node_ip: 202.117.179.205\n",
      "  pid: 20202\n",
      "  time_since_restore: 1.8382647037506104\n",
      "  time_this_iter_s: 0.018158674240112305\n",
      "  time_total_s: 1.8382647037506104\n",
      "  timestamp: 1614425394\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 10\n",
      "  trial_id: '00000'\n",
      "  \n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "== Status ==<br>Memory usage on this node: 18.2/503.2 GiB<br>Using AsyncHyperBand: num_stopped=2\nBracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: -0.563488 | Iter 1.000: -0.6555925<br>Resources requested: 0/48 CPUs, 0/1 GPUs, 0.0/333.79 GiB heap, 0.0/101.46 GiB objects<br>Result logdir: /home/xuhaowei/ray_results/train_breast_cancer<br>Number of trials: 3 (3 TERMINATED)<br><table>\n<thead>\n<tr><th>Trial name               </th><th>status    </th><th>loc  </th><th style=\"text-align: right;\">        eta</th><th style=\"text-align: right;\">      gamma</th><th>grow_policy  </th><th style=\"text-align: right;\">  max_depth</th><th style=\"text-align: right;\">     acc</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th></tr>\n</thead>\n<tbody>\n<tr><td>train_breast_cancer_00000</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">0.0363721  </td><td style=\"text-align: right;\">1.70701e-05</td><td>depthwise    </td><td style=\"text-align: right;\">          4</td><td style=\"text-align: right;\">0.937063</td><td style=\"text-align: right;\">    10</td><td style=\"text-align: right;\">        1.83826 </td></tr>\n<tr><td>train_breast_cancer_00001</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">0.0111021  </td><td style=\"text-align: right;\">0.00706989 </td><td>lossguide    </td><td style=\"text-align: right;\">          5</td><td style=\"text-align: right;\">        </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.79318 </td></tr>\n<tr><td>train_breast_cancer_00002</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">0.000227598</td><td style=\"text-align: right;\">4.42244e-07</td><td>lossguide    </td><td style=\"text-align: right;\">          8</td><td style=\"text-align: right;\">        </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.769577</td></tr>\n</tbody>\n</table><br><br>"
     },
     "metadata": {}
    }
   ],
   "source": [
    "tune_model = tune.run(\n",
    "    train_breast_cancer,  # 已定义好的模型结构\n",
    "    resources_per_trial={'cpu': 2},  # 每轮使用cpu的数量\n",
    "    config=config,  # 参数空间\n",
    "    num_samples=3,  # 运行Trails的（外层的）次数\n",
    "    # 超参数优化器 ASHAScheduler：异步Successive Halving优化算法的实现\n",
    "    # log损失，metric按min方向优化\n",
    "    scheduler=ASHAScheduler(metric='eval-logloss', mode='min')\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Best config is:  {'verbosity': 0, 'num_threads': 2, 'objective': 'binary:logistic', 'booster': 'gbtree', 'eval_metric': ['auc', 'ams@0', 'logloss'], 'max_depth': 8, 'eta': 0.00022759836097495771, 'gamma': 4.42244480744221e-07, 'grow_policy': 'lossguide'}\n"
     ]
    }
   ],
   "source": [
    "print('Best config is: ', tune_model.get_best_config(metric='eval-logloss'))"
   ]
  }
 ]
}